# coding:utf-8
# ip, time_local, http_method, url, http_status, body_bytes_sent , http_referer, ua
import re
from datetime import datetime

logline = '''183.60.212.153 - - [19/Feb/2013:10:23:29 +0800] "GET /o2o/media.html?menu=3 HTTP/1.1" 200 16691 "-" "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"'''

ops = {
    # 'datetime': lambda timestr: datetime.strptime(timestr, "%d/%b/%Y:%H:%M:%S %z"),
    'request': lambda request: dict(zip(('method', 'url', 'protocol'), request.split())),
    'status': int,
    'size': int
}


def extract(line):
    pattern = '''(?P<remote_addr>[\d\.]{7,}) - - (?:\[(?P<datetime>[^\[\]]+)\]) "(?P<request>[^"]+)" (?P<status>\d+) (?P<size>\d+) "(?P<refer>[^"]+)" "(?P<user_agent>[^"]+)"'''
    regex = re.compile(pattern)
    matcher = regex.match(line)
    if matcher:
        return {k: ops.get(k, lambda x: x)(v) for k, v in matcher.groupdict().items()}
    else:
        raise Exception('No match')


def get_nginx_log_data(file_path="nginx/access_sum_cp.log"):
    # line_num = 0
    nginx_log_list = list()
    with open(file_path, "r") as ngfile:
        for line in ngfile:
            one_item = dict()

            _nodes = line.split()
            IP = _nodes[0]
            # Time = _nodes[3][1:-1].replace(":", " ", 1)  # 将时间转换为17/Jun/2017 12:43:4格式
            Time = _nodes[3][1:-1]
            Methods = _nodes[5][1:]
            Source = _nodes[6]
            Protocol = _nodes[7][:-1]
            Status = _nodes[8]
            body_size = _nodes[9]

            # print(len(_nodes))

            if len(_nodes) > 10:
                http_refer = _nodes[10].replace('"', '')

                if len(_nodes) > 11:
                    user_agent = ' '.join(_nodes[11:-1]).split('"')[-1]
                else:
                    user_agent = '-'
            else:
                http_refer = '-'
            # http_referer = _nodes[10]
            # print(IP, Time, Methods, Source, Protocol, Status, body_size, http_refer, user_agent)

            one_item['ip'] = IP
            one_item['visit_time'] = Time
            one_item['request_method'] = Methods
            one_item['source_url'] = Source
            one_item['protocol'] = Protocol
            one_item['status'] = Status
            one_item['body_size'] = body_size
            one_item['http_refer'] = http_refer
            one_item['user_agent'] = user_agent

            nginx_log_list.append(one_item)

    return nginx_log_list


def get_nginx_log_data_01(file_path="nginx/access_sum_cp.log"):
    '''
    :param file_path:
    :return: 清洗完成的nginx数据
    '''
    nginx_log_list = list()
    line_num = 0
    with open(file_path, "r", encoding='utf-8') as ngfile:
        lines = ngfile.readlines()
        for line in lines:
            one_item = dict()
            line_num += 1
            try:
                one_result = extract(str(line))
                one_item['ip'] = one_result.get('remote_addr')
                one_item['visit_time'] = one_result.get('datetime')
                one_item['request_method'] = one_result.get('request').get('method')
                one_item['source_url'] = one_result.get('request').get('url')
                one_item['protocol'] = one_result.get('request').get('protocol')
                one_item['status'] = one_result.get('status')
                one_item['body_size'] = one_result.get('size')
                one_item['http_refer'] = one_result.get('refer')
                one_item['user_agent'] = one_result.get('user_agent')

                nginx_log_list.append(one_item)

            except Exception as e:
                print('在{}行出错了'.format(line_num))
                print(e)
    return nginx_log_list


def get_nginx_log_data_02(file_path="nginx/access_sum_cp.log"):
    nginx_log_list = list()
    line_num = 0
    with open(file_path, "r", encoding='utf-8') as ngfile:
        lines = ngfile.readlines()
        for line in lines:
            line_num += 1
            try:
                nginx_log_list.append(line)
            except Exception as e:
                print('在{}行出错了'.format(line_num))
                print(e)
    return nginx_log_list


if __name__ == '__main__':
    # data_list = get_nginx_log_data_01()
    # log_pro = extract(logline)
    # print(log_pro)
    # for item in data_list:
    #     print(item)

    # one_line = '''114.86.166.63 - - [30/Dec/2019:09:42:30 +0800] "GET / HTTP/1.1" 400 182 "-" "-"'''
    # one_line_01 = '''45.79.152.7 - - [30/Dec/2019:09:27:18 +0800] "\x16\x03\x01\x00\xF5\x01\x00\x00\xF1\x03\x03P\x04e\x99RN6\xB2\x03\xC2O\x94&\x94bf&\xCE\x8A\x10\x8D'd\x7FE\x15\x1E\x18h\xFEc\x1B\x00\x00\x82\x003\x009\x005\x00/\xC0,\xC00\x00\xA3\x00\x9F\xCC\xA9\xCC\xA8\xCC\xAA\xC0\xAF\xC0\xAD\xC0\xA3\xC0\x9F\xC0+\xC0/\x00\xA2\x00\x9E\xC0\xAE\xC0\xAC\xC0\xA2\xC0\x9E\xC0$\xC0(\x00k\x00j\xC0s\xC0w\x00\xC4\x00\xC3\xC0#\xC0'\x00g\x00@\xC0r\xC0v\x00\xBE\x00\xBD\xC0" 400 182 "-" "-"'''
    # result = extract(one_line_01)
    # print(result)

    one_line = '''61.219.11.153 - - [26/Dec/2019:03:19:56 +0800] "\x01\x00\x00\x00" 400 182 "-" "-"'''
    result = extract(one_line)
    print(result)
    # data_list = get_nginx_log_data_01()
    # for item in data_list:
    #     print(item)
    # print(len(data_list))